import mlflow
import pandas as pd
def generate_recommendations_with_comparision(
experiment_ids,
aggregation_function="common_features",
main_note="sizes_acts",
note_to_compare="sizes_L2_without_acts",
group_type="sim"
):
all_rows = []
for exp_id in experiment_ids:
runs = mlflow.search_runs(
experiment_ids=[exp_id],
output_format="list"
)
for run in runs:
note = run.data.params.get("note")
fusion = run.data.params.get("SAE_fusion_strategy")
group = run.data.params.get("group_type")
if fusion != aggregation_function or group != group_type:
continue
dataset = run.data.params.get("dataset", f"Exp-{exp_id}")
dim = int(run.data.params.get("embedding_dim", 0))
topk = int(run.data.params.get("top_k", 0))
row_key = (dataset, dim, topk)
model_type = "main" if note == main_note else "compare" if note == note_to_compare else None
if not model_type:
continue
metrics = {
("G/mean", model_type): run.data.metrics.get("CommonItemsNDCG20/median"),
("U/mean", model_type): run.data.metrics.get("NDCG20/mean"),
("U/min", model_type): run.data.metrics.get("NDCG20/min"),
("Pop", model_type): run.data.metrics.get("Popularity/mean"),
}
all_rows.append((row_key, metrics))
# Combine metrics into a dictionary
records = {}
for key, metrics in all_rows:
if key not in records:
records[key] = {}
records[key].update(metrics)
df = pd.DataFrame.from_dict(records, orient="index")
df.index.names = ["Dataset", "Dimensions", "TopK"]
# Calculate % differences
result_cols = []
for metric in sorted(set(k[0] for k in df.columns)):
main_col = (metric, "main")
compare_col = (metric, "compare")
percent_col = (metric, "% change")
if main_col in df.columns and compare_col in df.columns:
df[percent_col] = ((df[main_col] - df[compare_col]) / df[compare_col].abs()) * 100
result_cols.extend([main_col, percent_col, compare_col])
elif main_col in df.columns:
result_cols.append(main_col)
# Keep only main and percent change columns, sort them by metric
df = df[result_cols]
df = df.round(2)
df = df.sort_index(axis=1, level=0).sort_values(by=["Dataset", "Dimensions", "TopK"])
return df.reset_index()
import mlflow
import pandas as pd
def generate_sae_table_with_comparison(experiment_ids, main_note, note_to_compare):
all_rows = []
for exp_id in experiment_ids:
runs = mlflow.search_runs(
experiment_ids=[exp_id],
output_format="list"
)
for run in runs:
note = run.data.params.get("note")
if note not in [main_note, note_to_compare]:
continue
dataset = run.data.params.get("dataset", f"Exp-{exp_id}")
dim = int(run.data.params.get("embedding_dim", 0))
base_factors = float(run.data.params.get("base_factors", 0.0))
expansion_ratio = dim / base_factors if base_factors else None
topk = int(run.data.params.get("top_k", 0))
row_key = (dim, round(expansion_ratio, 3), topk)
model_type = "main" if note == main_note else "compare"
metrics = {
(dataset, model_type, "CS"): run.data.metrics.get("CosineSim/test"),
(dataset, model_type, "Deg"): run.data.metrics.get("NDCG20_Degradation/test"),
(dataset, model_type, "Deads"): run.data.metrics.get("DeadNeurons/test")
}
all_rows.append((row_key, metrics))
# Build DataFrame
records = {}
for key, metrics in all_rows:
if key not in records:
records[key] = {}
records[key].update(metrics)
df = pd.DataFrame.from_dict(records, orient="index")
df.index.names = ["Dimensions", "Expansion Ratio", "TopK"]
# Calculate % differences and arrange columns
result_cols = []
for (dataset, metric) in sorted(set((k[0], k[2]) for k in df.columns)):
main_col = (dataset, "main", metric)
compare_col = (dataset, "compare", metric)
percent_col = (dataset, "% change", metric)
if main_col in df.columns and compare_col in df.columns:
df[percent_col] = ((df[main_col] - df[compare_col]) / df[compare_col].abs()) * 100
result_cols.extend([main_col, percent_col, compare_col])
elif main_col in df.columns:
result_cols.append(main_col)
# Finalize view: sort columns, round values
df = df[result_cols]
df = df.round(3)
df = df.sort_index(axis=1, level=[0, 2]).sort_values(
by=["Dimensions", "Expansion Ratio", "TopK"],
ascending=[True, True, True]
)
return df.reset_index()
5. Contrastive Loss¶
Ackkoliv podle predchozich pruzkumu ukazuji autoencodery na specificke rysy, trenink probiha bez jakekoliv o to to zajistit. Vyzkum ... ukazuje metodu ...
Pojdme si ukazat jak se to promitne na vysledcich
Table of reconstrictions metrics¶
compare = L2, main = With contrastive loss in ration 0.7/0.3
- CS - Cosine Similarity of original and reconstructed embeddings
- Deads - Percentage of dead neurons in the sparse embedding
- Deg - Degradation of NDCG between ELSA model and ELSA + Autoencoder
sae_experiments = ['657713966175362303', '852893065079987597']
table = generate_sae_table_with_comparison(
sae_experiments,
main_note="sizes_constrastive_3",
note_to_compare="sizes_L2",
)
table
| Dimensions | Expansion Ratio | TopK | LastFM1k | MovieLens | |||||||||||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| % change | compare | main | % change | compare | main | % change | ... | main | % change | compare | main | % change | compare | main | % change | compare | main | ||||
| CS | CS | CS | Deads | Deads | Deads | Deg | ... | Deg | CS | CS | CS | Deads | Deads | Deads | Deg | Deg | Deg | ||||
| 0 | 1024 | 4.0 | 32 | 0.328 | 0.903 | 0.906 | 45.217 | 0.225 | 0.326 | 3.573 | ... | -0.013 | -1.269 | 0.947 | 0.935 | NaN | 0.000 | 0.000 | -39.199 | -0.012 | -0.017 |
| 1 | 1024 | 4.0 | 64 | 0.204 | 0.926 | 0.927 | 30.588 | 0.083 | 0.108 | -24.729 | ... | -0.008 | -1.404 | 0.970 | 0.956 | NaN | 0.000 | 0.000 | -58.636 | -0.006 | -0.009 |
| 2 | 1024 | 4.0 | 128 | 0.022 | 0.949 | 0.949 | -7.692 | 0.013 | 0.012 | 36.642 | ... | -0.005 | -1.764 | 0.991 | 0.974 | NaN | 0.000 | 0.000 | -146.282 | -0.002 | -0.006 |
| 3 | 2048 | 8.0 | 32 | 0.265 | 0.902 | 0.905 | 10.275 | 0.532 | 0.587 | -334.711 | ... | -0.009 | -0.657 | 0.943 | 0.936 | inf | 0.000 | 0.005 | 0.416 | -0.014 | -0.014 |
| 4 | 2048 | 8.0 | 64 | 0.128 | 0.925 | 0.926 | 39.300 | 0.251 | 0.350 | 13.408 | ... | -0.013 | -1.104 | 0.966 | 0.955 | NaN | 0.000 | 0.000 | -28.350 | -0.007 | -0.009 |
| 5 | 2048 | 8.0 | 128 | 0.040 | 0.950 | 0.951 | 46.970 | 0.097 | 0.142 | 30.588 | ... | -0.013 | -1.553 | 0.987 | 0.971 | NaN | 0.000 | 0.000 | -148.760 | -0.002 | -0.006 |
| 6 | 4096 | 16.0 | 32 | 0.326 | 0.898 | 0.901 | 4.632 | 0.743 | 0.778 | 9.271 | ... | -0.002 | -0.616 | 0.941 | 0.935 | 310.714 | 0.007 | 0.028 | 4.327 | -0.015 | -0.015 |
| 7 | 4096 | 16.0 | 64 | -0.036 | 0.924 | 0.923 | 9.235 | 0.539 | 0.589 | 1454.409 | ... | 0.004 | -0.763 | 0.961 | 0.953 | inf | 0.000 | 0.001 | -2.511 | -0.009 | -0.009 |
| 8 | 4096 | 16.0 | 128 | -0.178 | 0.950 | 0.948 | 22.482 | 0.303 | 0.371 | 28.993 | ... | -0.006 | -0.835 | 0.978 | 0.970 | NaN | 0.000 | 0.000 | -27.357 | -0.004 | -0.006 |
9 rows × 21 columns
table.mean()
Dimensions 2389.333333
Expansion Ratio 9.333333
TopK 74.666667
LastFM1k % change CS 0.122111
compare CS 0.925222
main CS 0.926222
% change Deads 22.334111
compare Deads 0.309556
main Deads 0.362556
% change Deg 135.271556
compare Deg -0.008111
main Deg -0.007222
MovieLens % change CS -1.107222
compare CS 0.964889
main CS 0.953889
% change Deads inf
compare Deads 0.000778
main Deads 0.003778
% change Deg -49.594667
compare Deg -0.007889
main Deg -0.010111
dtype: float64
- zvysuje se pocet dead neurons
- jinak hodnoty jsou si hodne podobny
Table of reconstrictions metrics¶
compare = L2, main = With contrastive loss in ration 0.5/0.5
- CS - Cosine Similarity of original and reconstructed embeddings
- Deads - Percentage of dead neurons in the sparse embedding
- Deg - Degradation of NDCG between ELSA model and ELSA + Autoencoder
sae_experiments = ['657713966175362303', '852893065079987597']
table = generate_sae_table_with_comparison(
sae_experiments,
main_note="sizes_constrastive_5",
note_to_compare="sizes_L2",
)
table
| Dimensions | Expansion Ratio | TopK | LastFM1k | MovieLens | |||||||||||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| % change | compare | main | % change | compare | main | % change | ... | main | % change | compare | main | % change | compare | main | % change | compare | main | ||||
| CS | CS | CS | Deads | Deads | Deads | Deg | ... | Deg | CS | CS | CS | Deads | Deads | Deads | Deg | Deg | Deg | ||||
| 0 | 1024 | 4.0 | 32 | 0.328 | 0.903 | 0.906 | 45.217 | 0.225 | 0.326 | 3.573 | ... | -0.013 | -1.269 | 0.947 | 0.935 | NaN | 0.000 | 0.000 | -39.199 | -0.012 | -0.017 |
| 1 | 1024 | 4.0 | 64 | 0.204 | 0.926 | 0.927 | 30.588 | 0.083 | 0.108 | -24.729 | ... | -0.008 | -1.404 | 0.970 | 0.956 | NaN | 0.000 | 0.000 | -58.636 | -0.006 | -0.009 |
| 2 | 1024 | 4.0 | 128 | 0.022 | 0.949 | 0.949 | -7.692 | 0.013 | 0.012 | 36.642 | ... | -0.005 | -1.764 | 0.991 | 0.974 | NaN | 0.000 | 0.000 | -146.282 | -0.002 | -0.006 |
| 3 | 2048 | 8.0 | 32 | 0.265 | 0.902 | 0.905 | 10.275 | 0.532 | 0.587 | -334.711 | ... | -0.009 | -0.657 | 0.943 | 0.936 | inf | 0.000 | 0.005 | 0.416 | -0.014 | -0.014 |
| 4 | 2048 | 8.0 | 64 | 0.128 | 0.925 | 0.926 | 39.300 | 0.251 | 0.350 | 13.408 | ... | -0.013 | -1.104 | 0.966 | 0.955 | NaN | 0.000 | 0.000 | -28.350 | -0.007 | -0.009 |
| 5 | 2048 | 8.0 | 128 | 0.040 | 0.950 | 0.951 | 46.970 | 0.097 | 0.142 | 30.588 | ... | -0.013 | -1.553 | 0.987 | 0.971 | NaN | 0.000 | 0.000 | -148.760 | -0.002 | -0.006 |
| 6 | 4096 | 16.0 | 32 | 0.326 | 0.898 | 0.901 | 4.632 | 0.743 | 0.778 | 9.271 | ... | -0.002 | -0.616 | 0.941 | 0.935 | 310.714 | 0.007 | 0.028 | 4.327 | -0.015 | -0.015 |
| 7 | 4096 | 16.0 | 64 | -0.036 | 0.924 | 0.923 | 9.235 | 0.539 | 0.589 | 1454.409 | ... | 0.004 | -0.763 | 0.961 | 0.953 | inf | 0.000 | 0.001 | -2.511 | -0.009 | -0.009 |
| 8 | 4096 | 16.0 | 128 | -0.178 | 0.950 | 0.948 | 22.482 | 0.303 | 0.371 | 28.993 | ... | -0.006 | -0.835 | 0.978 | 0.970 | NaN | 0.000 | 0.000 | -27.357 | -0.004 | -0.006 |
9 rows × 21 columns
table.mean()
Dimensions 2389.333333
Expansion Ratio 9.333333
TopK 74.666667
LastFM1k % change CS 0.122111
compare CS 0.925222
main CS 0.926222
% change Deads 22.334111
compare Deads 0.309556
main Deads 0.362556
% change Deg 135.271556
compare Deg -0.008111
main Deg -0.007222
MovieLens % change CS -1.107222
compare CS 0.964889
main CS 0.953889
% change Deads inf
compare Deads 0.000778
main Deads 0.003778
% change Deg -49.594667
compare Deg -0.007889
main Deg -0.010111
dtype: float64
Temer totozne vysledky
- pro CS skoro stejne
- pro deg pro last lepsi, pro echo nest trochu horsi
SAE group recommendation performance for common features aggregation function and similar groups¶
Comparing the basic variant (compare) with the contrastive loss coefficient 0.3
experiments = ['523100174176986081', '333391697323445885']
table = generate_recommendations_with_comparision(
experiments,
aggregation_function="common_features",
main_note="sizes_constrastive_3_without_acts",
note_to_compare="sizes_L2_without_acts",
group_type="sim",
)
table
| Dataset | Dimensions | TopK | G/mean | Pop | U/mean | U/min | |||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| % change | compare | main | % change | compare | main | % change | compare | main | % change | compare | main | ||||
| 0 | LastFM1k | 1024 | 32 | -3.24 | 0.59 | 0.57 | -0.07 | 0.62 | 0.62 | 1.36 | 0.79 | 0.80 | -3.85 | 0.63 | 0.61 |
| 1 | LastFM1k | 1024 | 64 | -1.61 | 0.58 | 0.57 | 2.59 | 0.61 | 0.63 | -0.71 | 0.80 | 0.80 | 0.83 | 0.63 | 0.63 |
| 2 | LastFM1k | 1024 | 128 | -6.06 | 0.60 | 0.57 | 1.02 | 0.60 | 0.61 | 1.10 | 0.80 | 0.81 | 0.32 | 0.63 | 0.63 |
| 3 | LastFM1k | 2048 | 32 | -4.40 | 0.59 | 0.56 | 0.92 | 0.64 | 0.65 | -0.22 | 0.79 | 0.79 | 1.48 | 0.62 | 0.63 |
| 4 | LastFM1k | 2048 | 64 | -0.01 | 0.60 | 0.60 | 2.58 | 0.62 | 0.63 | 0.64 | 0.81 | 0.81 | 0.30 | 0.64 | 0.64 |
| 5 | LastFM1k | 2048 | 128 | 3.55 | 0.61 | 0.63 | -0.01 | 0.61 | 0.61 | 1.11 | 0.81 | 0.82 | 6.52 | 0.61 | 0.65 |
| 6 | LastFM1k | 4096 | 32 | -0.98 | 0.57 | 0.57 | -0.11 | 0.66 | 0.66 | 0.77 | 0.78 | 0.79 | -0.47 | 0.63 | 0.62 |
| 7 | LastFM1k | 4096 | 64 | 12.90 | 0.52 | 0.59 | 1.80 | 0.63 | 0.64 | -0.88 | 0.81 | 0.80 | -2.17 | 0.62 | 0.61 |
| 8 | LastFM1k | 4096 | 128 | 0.33 | 0.59 | 0.60 | 0.74 | 0.62 | 0.62 | -0.55 | 0.81 | 0.81 | -0.48 | 0.63 | 0.63 |
| 9 | MovieLens | 1024 | 32 | -65.54 | 0.58 | 0.20 | -26.18 | 0.52 | 0.38 | -13.04 | 0.64 | 0.56 | -19.32 | 0.52 | 0.42 |
| 10 | MovieLens | 1024 | 64 | -29.43 | 0.54 | 0.38 | -20.79 | 0.50 | 0.40 | -10.15 | 0.65 | 0.59 | -19.52 | 0.53 | 0.42 |
| 11 | MovieLens | 1024 | 128 | -32.27 | 0.65 | 0.44 | -22.83 | 0.49 | 0.38 | -11.46 | 0.67 | 0.59 | -11.32 | 0.51 | 0.45 |
| 12 | MovieLens | 2048 | 32 | -39.04 | 0.68 | 0.42 | -14.04 | 0.49 | 0.42 | -11.75 | 0.66 | 0.58 | -24.96 | 0.54 | 0.40 |
| 13 | MovieLens | 2048 | 64 | -2.75 | 0.45 | 0.44 | -13.99 | 0.48 | 0.41 | -6.44 | 0.65 | 0.60 | -14.93 | 0.53 | 0.45 |
| 14 | MovieLens | 2048 | 128 | -14.33 | 0.51 | 0.44 | -19.76 | 0.48 | 0.38 | -14.63 | 0.67 | 0.57 | -22.67 | 0.53 | 0.41 |
| 15 | MovieLens | 4096 | 32 | -7.52 | 0.66 | 0.61 | -15.29 | 0.48 | 0.41 | -12.17 | 0.66 | 0.58 | -24.22 | 0.55 | 0.42 |
| 16 | MovieLens | 4096 | 64 | -22.57 | 0.59 | 0.46 | -14.65 | 0.51 | 0.43 | -6.25 | 0.66 | 0.62 | -13.86 | 0.53 | 0.46 |
| 17 | MovieLens | 4096 | 128 | -43.30 | 0.56 | 0.32 | -26.10 | 0.49 | 0.36 | -16.70 | 0.65 | 0.54 | -25.24 | 0.51 | 0.38 |
SAE group recommendation performance for common features aggregation function and similar groups¶
Comparing the basic variant (compare) with the contrastive loss coefficient 0.5
experiments = ['523100174176986081', '333391697323445885']
generate_recommendations_with_comparision(
experiments,
aggregation_function="common_features",
main_note="sizes_constrastive_5_without_acts",
note_to_compare="sizes_L2_without_acts",
group_type="sim",
)
| Dataset | Dimensions | TopK | G/mean | Pop | U/mean | U/min | |||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| % change | compare | main | % change | compare | main | % change | compare | main | % change | compare | main | ||||
| 0 | LastFM1k | 1024 | 32 | -1.43 | 0.59 | 0.58 | 3.28 | 0.62 | 0.64 | 1.12 | 0.79 | 0.80 | -2.74 | 0.63 | 0.62 |
| 1 | LastFM1k | 1024 | 64 | 1.89 | 0.58 | 0.59 | 3.27 | 0.61 | 0.63 | -0.29 | 0.80 | 0.80 | 2.13 | 0.63 | 0.64 |
| 2 | LastFM1k | 1024 | 128 | -6.56 | 0.60 | 0.56 | 1.40 | 0.60 | 0.61 | 0.08 | 0.80 | 0.80 | 0.97 | 0.63 | 0.63 |
| 3 | LastFM1k | 2048 | 32 | -0.03 | 0.59 | 0.59 | 1.49 | 0.64 | 0.65 | 1.08 | 0.79 | 0.80 | 1.37 | 0.62 | 0.63 |
| 4 | LastFM1k | 2048 | 64 | -4.12 | 0.60 | 0.57 | 3.26 | 0.62 | 0.64 | 0.59 | 0.81 | 0.81 | 0.65 | 0.64 | 0.64 |
| 5 | LastFM1k | 2048 | 128 | 0.30 | 0.61 | 0.61 | 2.60 | 0.61 | 0.63 | 0.54 | 0.81 | 0.81 | 4.69 | 0.61 | 0.64 |
| 6 | LastFM1k | 4096 | 32 | -3.28 | 0.57 | 0.56 | -0.56 | 0.66 | 0.65 | 1.80 | 0.78 | 0.80 | -1.81 | 0.63 | 0.61 |
| 7 | LastFM1k | 4096 | 64 | 26.63 | 0.52 | 0.66 | 3.47 | 0.63 | 0.65 | -0.99 | 0.81 | 0.80 | 2.00 | 0.62 | 0.64 |
| 8 | LastFM1k | 4096 | 128 | 5.81 | 0.59 | 0.63 | 2.28 | 0.62 | 0.63 | 0.62 | 0.81 | 0.82 | -1.38 | 0.63 | 0.62 |
| 9 | MovieLens | 1024 | 32 | -23.74 | 0.58 | 0.44 | -26.69 | 0.52 | 0.38 | -10.36 | 0.64 | 0.58 | -19.76 | 0.52 | 0.42 |
| 10 | MovieLens | 1024 | 64 | -18.55 | 0.54 | 0.44 | -28.19 | 0.50 | 0.36 | -16.61 | 0.65 | 0.55 | -25.43 | 0.53 | 0.39 |
| 11 | MovieLens | 1024 | 128 | -41.46 | 0.65 | 0.38 | -25.41 | 0.49 | 0.37 | -16.39 | 0.67 | 0.56 | -18.58 | 0.51 | 0.41 |
| 12 | MovieLens | 2048 | 32 | -81.47 | 0.68 | 0.13 | -17.79 | 0.49 | 0.40 | -18.98 | 0.66 | 0.53 | -31.13 | 0.54 | 0.37 |
| 13 | MovieLens | 2048 | 64 | -7.38 | 0.45 | 0.42 | -16.49 | 0.48 | 0.40 | -17.72 | 0.65 | 0.53 | -28.68 | 0.53 | 0.38 |
| 14 | MovieLens | 2048 | 128 | -80.74 | 0.51 | 0.10 | -22.20 | 0.48 | 0.37 | -16.58 | 0.67 | 0.56 | -22.75 | 0.53 | 0.41 |
| 15 | MovieLens | 4096 | 32 | -18.76 | 0.66 | 0.54 | -15.53 | 0.48 | 0.41 | -23.79 | 0.66 | 0.50 | -34.64 | 0.55 | 0.36 |
| 16 | MovieLens | 4096 | 64 | -88.04 | 0.59 | 0.07 | -26.17 | 0.51 | 0.38 | -21.52 | 0.66 | 0.52 | -30.79 | 0.53 | 0.37 |
| 17 | MovieLens | 4096 | 128 | 4.20 | 0.56 | 0.59 | -20.71 | 0.49 | 0.39 | -13.57 | 0.65 | 0.56 | -18.72 | 0.51 | 0.42 |
table.groupby("Dataset").mean().round(2)
| Dimensions | TopK | G/mean | Pop | U/mean | U/min | |||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| % change | compare | main | % change | compare | main | % change | compare | main | % change | compare | main | |||
| Dataset | ||||||||||||||
| LastFM1k | 2389.33 | 74.67 | -3.8 | 0.58 | 0.56 | -0.18 | 0.61 | 0.61 | -0.13 | 0.82 | 0.81 | -0.44 | 0.65 | 0.65 |
| MovieLens | 2389.33 | 74.67 | -7.5 | 0.66 | 0.61 | -2.53 | 0.53 | 0.52 | 0.12 | 0.69 | 0.69 | -0.36 | 0.58 | 0.58 |
SAE group recommendation performance for average aggregation function and similar groups¶
Comparing the basic variant (compare) with the contrastive loss coefficient 0.3
experiments = ['523100174176986081', '333391697323445885']
table = generate_recommendations_with_comparision(
experiments,
aggregation_function="average",
main_note="sizes_constrastive_3_with_acts",
note_to_compare="sizes_L2_with_acts",
group_type="sim",
)
table
| Dataset | Dimensions | TopK | G/mean | Pop | U/mean | U/min | |||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| % change | compare | main | % change | compare | main | % change | compare | main | % change | compare | main | ||||
| 0 | LastFM1k | 1024 | 32 | -4.71 | 0.52 | 0.49 | 0.96 | 0.60 | 0.61 | 0.44 | 0.81 | 0.81 | 1.51 | 0.64 | 0.64 |
| 1 | LastFM1k | 1024 | 64 | -3.70 | 0.56 | 0.54 | -1.28 | 0.61 | 0.60 | -0.26 | 0.81 | 0.81 | -3.06 | 0.65 | 0.63 |
| 2 | LastFM1k | 1024 | 128 | 1.30 | 0.57 | 0.58 | 0.76 | 0.61 | 0.61 | -1.06 | 0.81 | 0.80 | -0.84 | 0.65 | 0.64 |
| 3 | LastFM1k | 2048 | 32 | -0.93 | 0.58 | 0.58 | -0.60 | 0.62 | 0.61 | -0.63 | 0.82 | 0.81 | 0.13 | 0.64 | 0.64 |
| 4 | LastFM1k | 2048 | 64 | -7.05 | 0.58 | 0.54 | 0.32 | 0.61 | 0.61 | -0.57 | 0.82 | 0.82 | -0.05 | 0.67 | 0.67 |
| 5 | LastFM1k | 2048 | 128 | 0.02 | 0.65 | 0.65 | 0.05 | 0.60 | 0.60 | 0.62 | 0.82 | 0.82 | -2.50 | 0.67 | 0.65 |
| 6 | LastFM1k | 4096 | 32 | -2.81 | 0.59 | 0.57 | -0.70 | 0.61 | 0.61 | 0.41 | 0.81 | 0.82 | -0.61 | 0.65 | 0.65 |
| 7 | LastFM1k | 4096 | 64 | -3.75 | 0.58 | 0.56 | 0.27 | 0.61 | 0.61 | 0.34 | 0.82 | 0.82 | 0.47 | 0.65 | 0.65 |
| 8 | LastFM1k | 4096 | 128 | -12.53 | 0.62 | 0.54 | -1.42 | 0.61 | 0.60 | -0.42 | 0.82 | 0.81 | 1.01 | 0.64 | 0.65 |
| 9 | MovieLens | 1024 | 32 | -7.16 | 0.59 | 0.55 | -0.95 | 0.53 | 0.53 | 0.09 | 0.69 | 0.69 | -0.19 | 0.58 | 0.58 |
| 10 | MovieLens | 1024 | 64 | -16.40 | 0.74 | 0.62 | -4.94 | 0.54 | 0.51 | 0.02 | 0.70 | 0.70 | 0.60 | 0.58 | 0.58 |
| 11 | MovieLens | 1024 | 128 | -14.40 | 0.66 | 0.57 | -2.49 | 0.53 | 0.52 | -0.68 | 0.69 | 0.69 | -0.51 | 0.58 | 0.57 |
| 12 | MovieLens | 2048 | 32 | -8.45 | 0.66 | 0.60 | -4.13 | 0.54 | 0.52 | 0.61 | 0.69 | 0.69 | -0.76 | 0.58 | 0.58 |
| 13 | MovieLens | 2048 | 64 | -18.19 | 0.67 | 0.55 | -3.34 | 0.53 | 0.52 | 0.05 | 0.69 | 0.69 | -0.31 | 0.58 | 0.58 |
| 14 | MovieLens | 2048 | 128 | -3.69 | 0.70 | 0.68 | -0.77 | 0.53 | 0.52 | -0.49 | 0.69 | 0.69 | -1.17 | 0.57 | 0.57 |
| 15 | MovieLens | 4096 | 32 | 3.06 | 0.57 | 0.59 | -4.71 | 0.54 | 0.52 | 1.06 | 0.68 | 0.69 | 0.97 | 0.56 | 0.57 |
| 16 | MovieLens | 4096 | 64 | -14.10 | 0.69 | 0.59 | -1.28 | 0.54 | 0.53 | 0.33 | 0.69 | 0.69 | -1.05 | 0.59 | 0.58 |
| 17 | MovieLens | 4096 | 128 | 11.84 | 0.68 | 0.77 | -0.16 | 0.53 | 0.53 | 0.08 | 0.69 | 0.69 | -0.85 | 0.58 | 0.58 |
table.groupby("Dataset").mean().round(2)
| Dimensions | TopK | G/mean | Pop | U/mean | U/min | |||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| % change | compare | main | % change | compare | main | % change | compare | main | % change | compare | main | |||
| Dataset | ||||||||||||||
| LastFM1k | 2389.33 | 74.67 | -3.8 | 0.58 | 0.56 | -0.18 | 0.61 | 0.61 | -0.13 | 0.82 | 0.81 | -0.44 | 0.65 | 0.65 |
| MovieLens | 2389.33 | 74.67 | -7.5 | 0.66 | 0.61 | -2.53 | 0.53 | 0.52 | 0.12 | 0.69 | 0.69 | -0.36 | 0.58 | 0.58 |
SAE group recommendation performance for average aggregation function and similar groups¶
Comparing the basic variant (compare) with the contrastive loss coefficient 0.5
experiments = ['523100174176986081', '333391697323445885']
table = generate_recommendations_with_comparision(
experiments,
aggregation_function="average",
main_note="sizes_constrastive_5_with_acts",
note_to_compare="sizes_L2_with_acts",
group_type="sim",
)
table
| Dataset | Dimensions | TopK | G/mean | Pop | U/mean | U/min | |||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| % change | compare | main | % change | compare | main | % change | compare | main | % change | compare | main | ||||
| 0 | LastFM1k | 1024 | 32 | 9.14 | 0.52 | 0.57 | 1.81 | 0.60 | 0.61 | 0.41 | 0.81 | 0.81 | -0.13 | 0.64 | 0.63 |
| 1 | LastFM1k | 1024 | 64 | -0.79 | 0.56 | 0.55 | -1.22 | 0.61 | 0.60 | -0.19 | 0.81 | 0.81 | -4.01 | 0.65 | 0.62 |
| 2 | LastFM1k | 1024 | 128 | 4.32 | 0.57 | 0.60 | -0.03 | 0.61 | 0.61 | -0.21 | 0.81 | 0.81 | 1.01 | 0.65 | 0.66 |
| 3 | LastFM1k | 2048 | 32 | 4.20 | 0.58 | 0.61 | -1.42 | 0.62 | 0.61 | -0.22 | 0.82 | 0.82 | 2.84 | 0.64 | 0.66 |
| 4 | LastFM1k | 2048 | 64 | 14.24 | 0.58 | 0.66 | -0.33 | 0.61 | 0.61 | -0.32 | 0.82 | 0.82 | -1.45 | 0.67 | 0.66 |
| 5 | LastFM1k | 2048 | 128 | -3.54 | 0.65 | 0.62 | 0.23 | 0.60 | 0.60 | 0.40 | 0.82 | 0.82 | -2.05 | 0.67 | 0.65 |
| 6 | LastFM1k | 4096 | 32 | 0.60 | 0.59 | 0.59 | 0.03 | 0.61 | 0.62 | -0.37 | 0.81 | 0.81 | 0.68 | 0.65 | 0.66 |
| 7 | LastFM1k | 4096 | 64 | -1.44 | 0.58 | 0.58 | -0.95 | 0.61 | 0.60 | 0.48 | 0.82 | 0.82 | 1.71 | 0.65 | 0.66 |
| 8 | LastFM1k | 4096 | 128 | -15.95 | 0.62 | 0.52 | 0.55 | 0.61 | 0.62 | -0.21 | 0.82 | 0.82 | 0.69 | 0.64 | 0.65 |
| 9 | MovieLens | 1024 | 32 | -5.87 | 0.59 | 0.56 | -1.08 | 0.53 | 0.53 | -0.15 | 0.69 | 0.69 | 1.22 | 0.58 | 0.59 |
| 10 | MovieLens | 1024 | 64 | -0.35 | 0.74 | 0.74 | -4.27 | 0.54 | 0.51 | -0.98 | 0.70 | 0.69 | -2.19 | 0.58 | 0.56 |
| 11 | MovieLens | 1024 | 128 | -15.79 | 0.66 | 0.56 | -4.06 | 0.53 | 0.51 | 0.41 | 0.69 | 0.69 | -0.31 | 0.58 | 0.57 |
| 12 | MovieLens | 2048 | 32 | -14.23 | 0.66 | 0.56 | -8.61 | 0.54 | 0.49 | -0.79 | 0.69 | 0.68 | -3.02 | 0.58 | 0.56 |
| 13 | MovieLens | 2048 | 64 | -14.13 | 0.67 | 0.57 | -2.86 | 0.53 | 0.52 | 0.06 | 0.69 | 0.69 | -0.14 | 0.58 | 0.58 |
| 14 | MovieLens | 2048 | 128 | -4.24 | 0.70 | 0.67 | -1.86 | 0.53 | 0.52 | 0.18 | 0.69 | 0.69 | 0.52 | 0.57 | 0.58 |
| 15 | MovieLens | 4096 | 32 | -1.90 | 0.57 | 0.56 | -5.13 | 0.54 | 0.52 | 0.28 | 0.68 | 0.68 | 3.05 | 0.56 | 0.58 |
| 16 | MovieLens | 4096 | 64 | -13.15 | 0.69 | 0.60 | -2.72 | 0.54 | 0.53 | 0.14 | 0.69 | 0.69 | -3.96 | 0.59 | 0.56 |
| 17 | MovieLens | 4096 | 128 | -0.76 | 0.68 | 0.68 | -1.56 | 0.53 | 0.52 | 0.19 | 0.69 | 0.69 | -1.34 | 0.58 | 0.58 |
- horsi
table.groupby("Dataset").mean().round(2)
| Dimensions | TopK | G/mean | Pop | U/mean | U/min | |||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| % change | compare | main | % change | compare | main | % change | compare | main | % change | compare | main | |||
| Dataset | ||||||||||||||
| LastFM1k | 2389.33 | 74.67 | 1.20 | 0.58 | 0.59 | -0.15 | 0.61 | 0.61 | -0.03 | 0.82 | 0.82 | -0.08 | 0.65 | 0.65 |
| MovieLens | 2389.33 | 74.67 | -7.82 | 0.66 | 0.61 | -3.57 | 0.53 | 0.52 | -0.07 | 0.69 | 0.69 | -0.69 | 0.58 | 0.57 |
SAE group recommendation performance for average aggregation function and random groups¶
Comparing the basic variant (compare) with the contrastive loss coefficient 0.3
experiments = ['523100174176986081', '333391697323445885']
table = generate_recommendations_with_comparision(
experiments,
aggregation_function="average",
main_note="sizes_constrastive_3_with_acts",
note_to_compare="sizes_L2_with_acts",
group_type="random",
)
table
| Dataset | Dimensions | TopK | G/mean | Pop | U/mean | U/min | |||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| % change | compare | main | % change | compare | main | % change | compare | main | % change | compare | main | ||||
| 0 | LastFM1k | 1024 | 32 | 2.92 | 0.40 | 0.41 | 0.13 | 0.66 | 0.66 | 0.40 | 0.75 | 0.75 | -1.64 | 0.56 | 0.55 |
| 1 | LastFM1k | 1024 | 64 | -3.79 | 0.43 | 0.41 | -0.45 | 0.66 | 0.66 | 0.48 | 0.75 | 0.75 | 0.87 | 0.55 | 0.55 |
| 2 | LastFM1k | 1024 | 128 | 20.73 | 0.36 | 0.44 | 0.05 | 0.66 | 0.66 | -0.50 | 0.75 | 0.75 | 2.85 | 0.56 | 0.57 |
| 3 | LastFM1k | 2048 | 32 | -1.72 | 0.50 | 0.49 | 0.48 | 0.66 | 0.66 | -1.55 | 0.76 | 0.75 | -2.65 | 0.59 | 0.58 |
| 4 | LastFM1k | 2048 | 64 | 18.57 | 0.42 | 0.50 | -0.03 | 0.66 | 0.66 | 1.31 | 0.76 | 0.77 | -1.08 | 0.58 | 0.58 |
| 5 | LastFM1k | 2048 | 128 | 2.36 | 0.50 | 0.51 | 0.20 | 0.66 | 0.66 | -0.71 | 0.77 | 0.76 | -0.13 | 0.58 | 0.58 |
| 6 | LastFM1k | 4096 | 32 | -3.55 | 0.36 | 0.35 | -0.62 | 0.67 | 0.66 | -0.52 | 0.76 | 0.76 | 3.67 | 0.57 | 0.59 |
| 7 | LastFM1k | 4096 | 64 | -18.80 | 0.36 | 0.29 | 0.77 | 0.66 | 0.66 | 0.95 | 0.75 | 0.76 | -1.55 | 0.58 | 0.57 |
| 8 | LastFM1k | 4096 | 128 | -17.30 | 0.35 | 0.29 | 0.15 | 0.66 | 0.66 | 0.77 | 0.76 | 0.76 | -0.82 | 0.56 | 0.55 |
| 9 | MovieLens | 1024 | 32 | 6.09 | 0.59 | 0.62 | -5.08 | 0.57 | 0.54 | -0.86 | 0.69 | 0.68 | -3.74 | 0.56 | 0.53 |
| 10 | MovieLens | 1024 | 64 | 7.79 | 0.61 | 0.66 | -3.78 | 0.57 | 0.55 | -0.94 | 0.69 | 0.69 | -1.14 | 0.55 | 0.55 |
| 11 | MovieLens | 1024 | 128 | 12.65 | 0.54 | 0.61 | -2.42 | 0.57 | 0.56 | -0.20 | 0.69 | 0.69 | -0.44 | 0.54 | 0.54 |
| 12 | MovieLens | 2048 | 32 | 5.24 | 0.63 | 0.66 | -4.77 | 0.57 | 0.55 | 1.70 | 0.69 | 0.70 | 0.03 | 0.55 | 0.55 |
| 13 | MovieLens | 2048 | 64 | 16.56 | 0.55 | 0.65 | -5.37 | 0.58 | 0.55 | 0.90 | 0.69 | 0.69 | 2.69 | 0.55 | 0.56 |
| 14 | MovieLens | 2048 | 128 | 5.83 | 0.62 | 0.65 | -2.77 | 0.57 | 0.55 | -1.45 | 0.69 | 0.68 | 0.31 | 0.56 | 0.56 |
| 15 | MovieLens | 4096 | 32 | 10.42 | 0.57 | 0.63 | -4.99 | 0.58 | 0.55 | -0.38 | 0.69 | 0.69 | 0.40 | 0.54 | 0.55 |
| 16 | MovieLens | 4096 | 64 | 23.20 | 0.54 | 0.67 | -1.13 | 0.57 | 0.56 | -0.40 | 0.69 | 0.69 | 5.26 | 0.53 | 0.56 |
| 17 | MovieLens | 4096 | 128 | 5.98 | 0.62 | 0.65 | -0.10 | 0.56 | 0.56 | 0.29 | 0.69 | 0.69 | 2.16 | 0.54 | 0.55 |
table.groupby("Dataset").mean().round(2)
| Dimensions | TopK | G/mean | Pop | U/mean | U/min | |||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| % change | compare | main | % change | compare | main | % change | compare | main | % change | compare | main | |||
| Dataset | ||||||||||||||
| LastFM1k | 2389.33 | 74.67 | -0.06 | 0.41 | 0.41 | 0.08 | 0.66 | 0.66 | 0.07 | 0.76 | 0.76 | -0.05 | 0.57 | 0.57 |
| MovieLens | 2389.33 | 74.67 | 10.42 | 0.59 | 0.64 | -3.38 | 0.57 | 0.55 | -0.15 | 0.69 | 0.69 | 0.61 | 0.55 | 0.55 |
SAE group recommendation performance for average aggregation function and div groups¶
Comparing the basic variant (compare) with the contrastive loss coefficient 0.3
experiments = ['523100174176986081', '333391697323445885']
table = generate_recommendations_with_comparision(
experiments,
aggregation_function="average",
main_note="sizes_constrastive_3_with_acts",
note_to_compare="sizes_L2_with_acts",
group_type="div",
)
table
| Dataset | Dimensions | TopK | G/mean | Pop | U/mean | U/min | |||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| % change | compare | main | % change | compare | main | % change | compare | main | % change | compare | main | ||||
| 0 | LastFM1k | 1024 | 32 | -5.59 | 0.50 | 0.47 | -3.37 | 0.63 | 0.61 | 1.49 | 0.67 | 0.68 | 2.65 | 0.46 | 0.47 |
| 1 | LastFM1k | 1024 | 64 | 9.25 | 0.46 | 0.51 | -1.16 | 0.61 | 0.60 | 0.29 | 0.66 | 0.66 | 1.42 | 0.44 | 0.45 |
| 2 | LastFM1k | 1024 | 128 | 5.88 | 0.41 | 0.44 | -0.65 | 0.60 | 0.60 | 1.73 | 0.66 | 0.67 | 2.59 | 0.42 | 0.43 |
| 3 | LastFM1k | 2048 | 32 | 25.18 | 0.43 | 0.54 | -1.37 | 0.62 | 0.61 | 1.21 | 0.68 | 0.69 | -1.22 | 0.48 | 0.48 |
| 4 | LastFM1k | 2048 | 64 | -0.79 | 0.45 | 0.44 | 0.18 | 0.61 | 0.61 | -0.40 | 0.67 | 0.67 | 4.00 | 0.46 | 0.48 |
| 5 | LastFM1k | 2048 | 128 | -20.24 | 0.47 | 0.37 | -1.66 | 0.61 | 0.60 | 0.75 | 0.66 | 0.66 | 2.22 | 0.44 | 0.45 |
| 6 | LastFM1k | 4096 | 32 | -2.22 | 0.52 | 0.50 | 0.07 | 0.62 | 0.62 | 0.13 | 0.68 | 0.68 | -1.15 | 0.47 | 0.46 |
| 7 | LastFM1k | 4096 | 64 | 19.94 | 0.47 | 0.56 | 0.08 | 0.61 | 0.61 | -0.68 | 0.68 | 0.67 | -1.07 | 0.46 | 0.46 |
| 8 | LastFM1k | 4096 | 128 | 2.28 | 0.45 | 0.46 | -0.21 | 0.61 | 0.61 | 1.46 | 0.66 | 0.67 | 1.46 | 0.45 | 0.45 |
| 9 | MovieLens | 1024 | 32 | -13.86 | 0.15 | 0.13 | -14.26 | 0.43 | 0.37 | -5.96 | 0.62 | 0.58 | -5.14 | 0.45 | 0.42 |
| 10 | MovieLens | 1024 | 64 | 16.10 | 0.13 | 0.15 | -13.21 | 0.42 | 0.36 | -4.18 | 0.62 | 0.60 | -12.13 | 0.46 | 0.41 |
| 11 | MovieLens | 1024 | 128 | 0.00 | 0.12 | 0.12 | -7.45 | 0.42 | 0.39 | -3.14 | 0.63 | 0.61 | -10.50 | 0.44 | 0.39 |
| 12 | MovieLens | 2048 | 32 | -42.19 | 0.15 | 0.09 | -18.21 | 0.44 | 0.36 | -3.92 | 0.63 | 0.60 | -5.13 | 0.43 | 0.41 |
| 13 | MovieLens | 2048 | 64 | 0.00 | 0.12 | 0.12 | -14.85 | 0.43 | 0.37 | -4.43 | 0.62 | 0.60 | -6.40 | 0.44 | 0.41 |
| 14 | MovieLens | 2048 | 128 | 0.00 | 0.13 | 0.13 | -5.99 | 0.40 | 0.37 | -0.22 | 0.61 | 0.61 | -2.26 | 0.42 | 0.41 |
| 15 | MovieLens | 4096 | 32 | 20.91 | 0.11 | 0.13 | -6.61 | 0.45 | 0.42 | -4.58 | 0.64 | 0.61 | -8.14 | 0.47 | 0.43 |
| 16 | MovieLens | 4096 | 64 | 16.06 | 0.10 | 0.12 | -4.71 | 0.45 | 0.42 | -2.43 | 0.63 | 0.62 | -3.55 | 0.46 | 0.44 |
| 17 | MovieLens | 4096 | 128 | -22.60 | 0.13 | 0.10 | 0.56 | 0.40 | 0.40 | -1.81 | 0.62 | 0.61 | 2.79 | 0.42 | 0.43 |
table.groupby("Dataset").mean().round(2)
| Dimensions | TopK | G/mean | Pop | U/mean | U/min | |||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| % change | compare | main | % change | compare | main | % change | compare | main | % change | compare | main | |||
| Dataset | ||||||||||||||
| LastFM1k | 2389.33 | 74.67 | 3.74 | 0.46 | 0.48 | -0.90 | 0.61 | 0.61 | 0.66 | 0.67 | 0.67 | 1.21 | 0.45 | 0.46 |
| MovieLens | 2389.33 | 74.67 | -2.84 | 0.13 | 0.12 | -9.41 | 0.43 | 0.38 | -3.41 | 0.62 | 0.60 | -5.61 | 0.44 | 0.42 |